import requests
from lxml import etree
import pandas as pd

num = 0
url = "https://book.douban.com/tag/%E5%B0%8F%E8%AF%B4?start={}&type=T"
headers = {"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64)"
                         " AppleWebKit/537.36 (KHTML, like Gecko)"
                         " Chrome/67.0.3396.99 Safari/537.36"}

index = 0
book_name_arr = [0] * 1500
book_grade_arr = [0] * 1500
book_author_arr = [0] * 1500

while num < 1500:
    url = url.format(num)
    html_str = requests.get(url, headers=headers).content.decode()
    ret1 = etree.HTML(html_str).xpath("//li[@class='subject-item']")
    try:
        for content in ret1:

            # story_dict["书名"] = content.xpath(".//div[@class='info']/h2/a/text()")[0].strip()
            # story_dict["封面图片"] = content.xpath(".//a[@class='nbg']/img/@src")[0]
            # story_dict["作者"] = content.xpath(".//div[@class='pub']/text()")[0].split("/")[0].strip()
            # story_dict["出版社"] = content.xpath(".//div[@class='pub']/text()")[0].split("/")[-3].strip()
            # story_dict["出版时间"] = content.xpath(".//div[@class='pub']/text()")[0].split("/")[-2].strip()
            # story_dict["售价"] = content.xpath(".//div[@class='pub']/text()")[0].split("/")[-1].strip()
            # story_dict["评价人数"] = content.xpath(".//span[@class='pl']/text()")[0].strip()
            # story_dict["评价人数"] = story_dict["评价人数"] if len(story_dict["评价人数"]) >0 else None

            book_name = ""
            book_name += content.xpath(".//div[@class='info']/h2/a/text()")[0].strip()
            grade = ""
            grade += content.xpath(".//span[@class='rating_nums']/text()")[0]
            author = content.xpath(".//div[@class='pub']/text()")[0].split("/")[0].strip()
            if index >= 1500:
                continue
            book_name_arr[index] = book_name
            book_grade_arr[index] = grade
            book_author_arr[index] = author
            index += 1
    except Exception as e:
        print(e)
    num += 20
# df = pd.DataFrame({"名称": book_name_arr, "作者编号": book_author_arr, "评分": book_grade_arr})
# df.to_excel("book.xlsx", index=False, columns=['名称', '作者编号', '评分'])

df1 = pd.DataFrame({"名称": book_name_arr, "作者编号": book_author_arr})
df1.to_excel("name.xlsx", index=False, columns=['名称', '作者编号'])

df2 = pd.DataFrame({"作者编号": book_author_arr, "评分": book_grade_arr})
df2.to_excel("grade.xlsx", index=False, columns=['作者编号', '评分'])
